#include "SplitTool.h"

SplitTool::~SplitTool()
{

}

unordered_set<string> SplitTool::StopWords(const string & filename)
{
    unordered_set<string> stopWords;
    ifstream ifs;
    ifs.open(filename);
    if(!ifs.is_open())
    {
        cerr << "无法打开文件: " << filename << "\n";
    }

    string word;
    while(getline(ifs, word))
    {
        word.erase(0, word.find_first_not_of(" \t\n\r"));
        word.erase(word.find_last_not_of(" \t\n\r")+1);
        stopWords.insert(word);
    }

    return stopWords;
}

vector<string> SplitTool::tokenize(const string & text)
{
    vector<string> words;
    string word;
    for (char c : text)
    {
        if (isalnum(c))
        {
            char ch = std::tolower(static_cast<unsigned char>(c));
            word += ch;
        }
        else
        {
            if (!word.empty())
            {
                words.push_back(word);
                word.clear();
            }
        }
    }

    if(!word.empty())
    {
        words.push_back(word);
    }
    return words;
}


vector<string> SplitTool::cut(const string &filename)
{
    static unordered_set<string> cnStopWords = StopWords("En_yuliao/stop_words_eng.txt");
    vector<string> AllWords;
    ifstream file;
    file.open(filename);
    if (!file.is_open())
    {
        cerr << "无法打开文件：" << filename << "\n";
        return AllWords;
    }

    string line;
    while (getline(file, line))
    {
        vector<string> words = tokenize(line);
        
        vector<string> finalWords;
        //去掉通用词
        for(auto & word : words)
        {
            if(cnStopWords.find(word) == cnStopWords.end())
            {
                finalWords.push_back(word);
            }
        }

        //去掉停用词后再添加到结果中去
        AllWords.insert(AllWords.end(), finalWords.begin(), finalWords.end());
    }
    
    file.close();
    return AllWords;
}
